import time

import requests
import json
import pymongo
client = pymongo.MongoClient()
db = client.get_database("cctv")
collection = db.get_collection("news")

count = 0
page = 1
while True:
	res = requests.get(f"https://news.cctv.com/2019/07/gaiban/cmsdatainterface/page/news_{page}.jsonp?cb=news")
	content = res.content.decode()[5:-1]
	data = json.loads(content)
	items = data['data']['list']
	count += len(items)
	print(f"第{page}页获取总共数据{count}个。。。。。。。。。")
	total = data['data']['total']
	for item in items:
		title = item['title']
		brief = item['brief']
		image_url = item['image']
		file_name = image_url.split('/')[-1]
		file_path = f"./images/{file_name}"
		image_res = requests.get(item['image'])

		collection.insert_one({
			"title": title,
			"brief": brief,
			"file_path": file_path
		})
		with open(file_path, "wb") as f:
			f.write(image_res.content)

	page += 1
	if count >= total:
		break
	time.sleep(1)



client.close()